import glob
import logging
import os.path

from gxl_ai_utils.utils import utils_file

def do_datahandle():
    """"""
    utils_file.do_config_root_logger()
    logging.info("开始sot数据转移")
    input_asr_list_path = "/home/work_nfs15/mcshao/workspace/4o/wenet_whisper_finetune/examples/wenetspeech/whisper/data/shards_train_list.txt"
    # input_sot_list_path = "/home/work_nfs13/yhdai/data/4o_data/SOT/shards_list.txt"
    output_dir = "/home/node54_tmpdata/xlgeng/asr_data_2w"
    exit_file_list = utils_file.do_get_list_for_wav_dir(output_dir, suffix='tar')
    print(len(exit_file_list))
    new_exit_file_list = []
    for file_path in exit_file_list:
        if os.path.exists(file_path) and utils_file.get_file_size(file_path) > 10:
            new_exit_file_list.append(file_path)

    input_file_list = utils_file.load_list_file_clean(input_asr_list_path)
    new_input_file_list = []
    for input_file_path in input_file_list:
        output_file_path_corresponding = utils_file.do_replace_dir(input_file_path, output_dir)
        if output_file_path_corresponding in new_exit_file_list:
            continue
        new_input_file_list.append(input_file_path)
    print(f'剩余数据的lens: {len(new_input_file_list)}')
    tmp_file_path = utils_file.do_get_fake_file()
    utils_file.write_list_to_file(new_input_file_list, tmp_file_path)
    utils_file.do_copy_files_by_manifest(tmp_file_path, output_dir, 'list', num_thread=10)


if __name__=="__main__":
    """"""
    do_datahandle()



